{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.5.1 召回环节评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.evaluation import generate_question_context_pairs\n",
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext,OpenAIEmbedding\n",
    "from llama_index.node_parser import SentenceSplitter\n",
    "from llama_index.llms import OpenAI\n",
    "import os\n",
    "from llama_index.evaluation import RetrieverEvaluator\n",
    "import pandas as pd \n",
    "\n",
    "os.environ[\"OPENAI_API_BASE\"] = \"xxx\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"xxx\"\n",
    "llm = OpenAI(model=\"gpt-35-turbo\")\n",
    "# 文本加载与切块，data文件夹中存放txt文件\n",
    "documents = SimpleDirectoryReader(\"./data/\").load_data()\n",
    "node_parser = SentenceSplitter(chunk_size=512)\n",
    "nodes = node_parser.get_nodes_from_documents(documents)\n",
    "# 设置文本块id\n",
    "for idx, node in enumerate(nodes):\n",
    "    node.id_ = f\"node_{idx}\"\n",
    "# 构建向量数据库\n",
    "embed_model = OpenAIEmbedding()\n",
    "service_context = ServiceContext.from_defaults(embed_model=embed_model)\n",
    "vector_index = VectorStoreIndex(nodes, service_context=service_context)\n",
    "retriever = vector_index.as_retriever(similarity_top_k=2)\n",
    "# 根据原始文本块生成问题\n",
    "qa_dataset = generate_question_context_pairs(\n",
    "    nodes, llm=llm, num_questions_per_chunk=2\n",
    ")\n",
    "# 定义评估器\n",
    "retriever_evaluator = RetrieverEvaluator.from_metric_names(\n",
    "    [\"mrr\", \"hit_rate\"], retriever=retriever\n",
    ")\n",
    "# 在所有生成的问题上评估召回效果\n",
    "eval_results = await retriever_evaluator.aevaluate_dataset(qa_dataset)\n",
    "result = []\n",
    "for eval_result in eval_results:\n",
    "    metric_dict = eval_result.metric_vals_dict\n",
    "    result.append(metric_dict)\n",
    "result_df = pd.DataFrame(result)\n",
    "hit_rate = result_df[\"hit_rate\"].mean()\n",
    "mrr = result_df[\"mrr\"].mean()\n",
    "print(\"hit_rate:\", hit_rate)\n",
    "print(\"mrr:\", mrr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.5.2 模型回答评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.evaluation import (\n",
    "    DatasetGenerator,\n",
    "    QueryResponseDataset,\n",
    ")\n",
    "from llama_index import ServiceContext\n",
    "from llama_index.node_parser import SentenceSplitter\n",
    "from llama_index.llms import OpenAI\n",
    "import os\n",
    "from llama_index import SimpleDirectoryReader\n",
    "os.environ[\"OPENAI_API_BASE\"] = \"xxx\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"xxx\"\n",
    "\n",
    "eval_service_context = ServiceContext.from_defaults(\n",
    "    llm=OpenAI(model=\"gpt-35-turbo\")\n",
    ")\n",
    "# 文本加载与切块，data文件夹中存放txt文件\n",
    "documents = SimpleDirectoryReader(\"./data/\").load_data()\n",
    "node_parser = SentenceSplitter(chunk_size=512)\n",
    "nodes = node_parser.get_nodes_from_documents(documents)\n",
    "# 定义问答数据生成器\n",
    "dataset_generator = DatasetGenerator(\n",
    "    nodes[:5],\n",
    "    service_context=eval_service_context,\n",
    "    show_progress=True,\n",
    "    num_questions_per_chunk=1,\n",
    ")\n",
    "# 生成问答对\n",
    "eval_dataset = await dataset_generator.agenerate_dataset_from_nodes(num=6)\n",
    "# 保存问答对\n",
    "eval_dataset.save_json(\"data/qa_dataset.json\")\n",
    "# 加载问答对\n",
    "eval_dataset = QueryResponseDataset.from_json(\"data/qa_dataset.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n",
    "import pandas as pd\n",
    "import os\n",
    "from ragas.metrics import faithfulness,answer_relevancy,context_relevancy\n",
    "from ragas.llama_index import evaluate\n",
    "import nest_asyncio\n",
    "nest_asyncio.apply()\n",
    "os.environ[\"OPENAI_API_BASE\"] = \"xxx\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"xxx\"\n",
    "\n",
    "documents = SimpleDirectoryReader(\"./data/nyc_wikipedia/\").load_data()\n",
    "vector_index = VectorStoreIndex.from_documents(\n",
    "    documents, service_context=ServiceContext.from_defaults(chunk_size=512)\n",
    ")\n",
    "query_engine = vector_index.as_query_engine()\n",
    "eval_questions = [\n",
    "    \"What is the population of New York City as of 2020?\",\n",
    "    \"Which borough of New York City has the highest population?\",\n",
    "    \"What is the economic significance of New York City?\",\n",
    "    \"How did New York City get its name?\",\n",
    "    \"What is the significance of the Statue of Liberty in New York City?\",\n",
    "]\n",
    "\n",
    "eval_answers = [\n",
    "    \"8,804,000\",  # incorrect answer\n",
    "    \"Queens\",  # incorrect answer\n",
    "    \"New York City's economic significance is vast, as it serves as the global financial capital, housing Wall Street and major financial institutions. Its diverse economy spans technology, media, healthcare, education, and more, making it resilient to economic fluctuations. NYC is a hub for international business, attracting global companies, and boasts a large, skilled labor force. Its real estate market, tourism, cultural industries, and educational institutions further fuel its economic prowess. The city's transportation network and global influence amplify its impact on the world stage, solidifying its status as a vital economic player and cultural epicenter.\",\n",
    "    \"New York City got its name when it came under British control in 1664. King Charles II of England granted the lands to his brother, the Duke of York, who named the city New York in his own honor.\",\n",
    "    \"The Statue of Liberty in New York City holds great significance as a symbol of the United States and its ideals of liberty and peace. It greeted millions of immigrants who arrived in the U.S. by ship in the late 19th and early 20th centuries, representing hope and freedom for those seeking a better life. It has since become an iconic landmark and a global symbol of cultural diversity and freedom.\",\n",
    "]\n",
    "\n",
    "eval_answers = [[a] for a in eval_answers]\n",
    "metrics = [faithfulness, answer_relevancy, context_relevancy]\n",
    "result = evaluate(query_engine, metrics, eval_questions, eval_answers)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
